{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#继续优化mnist\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-6d2718fffa9f>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/liuqihan/.local/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/liuqihan/.local/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/liuqihan/.local/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/liuqihan/.local/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/liuqihan/.local/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "#import data\n",
    "data_dir ='/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir,one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#每个批次的大小\n",
    "batch_size = 100\n",
    "#计算一共有多少个批次\n",
    "n_batch = mnist.train.num_examples // batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#define learning rate\n",
    "lr = tf.Variable(0.001, dtype=tf.float32)\n",
    "#define dropout rate(last layer)\n",
    "keep_prob = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creat model\n",
    "\n",
    "#input layer\n",
    "x = tf.placeholder(tf.float32, [None,784])\n",
    "y = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hidden layer 1(尝试使用截断正态分布初始化权重)\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.tanh(tf.matmul(x, W1) + b1)\n",
    "#L1_drop = tf.nn.dropout(L1,keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hidden layer 2\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 300], stddev=0.1)) \n",
    "b2 = tf.Variable(tf.zeros([300]) + 0.1) \n",
    "L2 = tf.nn.tanh(tf.matmul(L1, W2) + b2)\n",
    "L2_drop = tf.nn.dropout(L2,keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#output layer \n",
    "W3 = tf.Variable(tf.truncated_normal([300, 10], stddev=0.1)) \n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1) \n",
    "y_ = tf.nn.softmax(tf.matmul(L2_drop, W3) + b3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-8-8caad342bbd7>:3: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#计算交叉熵\n",
    "cross_entropy= tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_)\n",
    ") \n",
    "\n",
    "#训练 \n",
    "train_step = tf.train.AdamOptimizer(lr).minimize(cross_entropy) \n",
    "\n",
    "#初始化变量 \n",
    "init = tf.global_variables_initializer()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#结果利用argmax返回最大值位置，并且计算准确率\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iterm0, Testing Accuracy0.9492, Learning_rate0.001\n",
      "Iterm1, Testing Accuracy0.9578, Learning_rate0.00095\n",
      "Iterm2, Testing Accuracy0.9648, Learning_rate0.0009025\n",
      "Iterm3, Testing Accuracy0.9671, Learning_rate0.000857375\n",
      "Iterm4, Testing Accuracy0.9682, Learning_rate0.00081450626\n",
      "Iterm5, Testing Accuracy0.9725, Learning_rate0.0007737809\n",
      "Iterm6, Testing Accuracy0.974, Learning_rate0.0007350919\n",
      "Iterm7, Testing Accuracy0.9758, Learning_rate0.0006983373\n",
      "Iterm8, Testing Accuracy0.9755, Learning_rate0.0006634204\n",
      "Iterm9, Testing Accuracy0.9742, Learning_rate0.0006302494\n",
      "Iterm10, Testing Accuracy0.9772, Learning_rate0.0005987369\n",
      "Iterm11, Testing Accuracy0.9783, Learning_rate0.0005688001\n",
      "Iterm12, Testing Accuracy0.9755, Learning_rate0.0005403601\n",
      "Iterm13, Testing Accuracy0.979, Learning_rate0.0005133421\n",
      "Iterm14, Testing Accuracy0.9803, Learning_rate0.000487675\n",
      "Iterm15, Testing Accuracy0.979, Learning_rate0.00046329122\n",
      "Iterm16, Testing Accuracy0.9802, Learning_rate0.00044012666\n",
      "Iterm17, Testing Accuracy0.9791, Learning_rate0.00041812033\n",
      "Iterm18, Testing Accuracy0.9778, Learning_rate0.00039721432\n",
      "Iterm19, Testing Accuracy0.9818, Learning_rate0.0003773536\n",
      "Iterm20, Testing Accuracy0.9807, Learning_rate0.00035848594\n",
      "Iterm21, Testing Accuracy0.9801, Learning_rate0.00034056162\n",
      "Iterm22, Testing Accuracy0.9815, Learning_rate0.00032353355\n",
      "Iterm23, Testing Accuracy0.9805, Learning_rate0.00030735688\n",
      "Iterm24, Testing Accuracy0.9812, Learning_rate0.000291989\n",
      "Iterm25, Testing Accuracy0.9821, Learning_rate0.00027738957\n",
      "Iterm26, Testing Accuracy0.9824, Learning_rate0.0002635201\n",
      "Iterm27, Testing Accuracy0.9823, Learning_rate0.00025034408\n",
      "Iterm28, Testing Accuracy0.9815, Learning_rate0.00023782688\n",
      "Iterm29, Testing Accuracy0.9815, Learning_rate0.00022593554\n",
      "Iterm30, Testing Accuracy0.9821, Learning_rate0.00021463877\n",
      "Iterm31, Testing Accuracy0.9821, Learning_rate0.00020390682\n",
      "Iterm32, Testing Accuracy0.9825, Learning_rate0.00019371149\n",
      "Iterm33, Testing Accuracy0.9826, Learning_rate0.0001840259\n",
      "Iterm34, Testing Accuracy0.9824, Learning_rate0.00017482461\n",
      "Iterm35, Testing Accuracy0.9824, Learning_rate0.00016608338\n",
      "Iterm36, Testing Accuracy0.9818, Learning_rate0.00015777921\n",
      "Iterm37, Testing Accuracy0.9831, Learning_rate0.00014989026\n",
      "Iterm38, Testing Accuracy0.9825, Learning_rate0.00014239574\n",
      "Iterm39, Testing Accuracy0.9839, Learning_rate0.00013527596\n",
      "Iterm40, Testing Accuracy0.9825, Learning_rate0.00012851215\n",
      "Iterm41, Testing Accuracy0.9836, Learning_rate0.00012208655\n",
      "Iterm42, Testing Accuracy0.9837, Learning_rate0.00011598222\n",
      "Iterm43, Testing Accuracy0.9841, Learning_rate0.00011018311\n",
      "Iterm44, Testing Accuracy0.9831, Learning_rate0.000104673956\n",
      "Iterm45, Testing Accuracy0.9838, Learning_rate9.944026e-05\n",
      "Iterm46, Testing Accuracy0.9838, Learning_rate9.446825e-05\n",
      "Iterm47, Testing Accuracy0.9831, Learning_rate8.974483e-05\n",
      "Iterm48, Testing Accuracy0.9839, Learning_rate8.525759e-05\n",
      "Iterm49, Testing Accuracy0.9838, Learning_rate8.099471e-05\n",
      "Iterm50, Testing Accuracy0.9834, Learning_rate7.6944976e-05\n"
     ]
    }
   ],
   "source": [
    "##################################\n",
    "#初始化\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    ##总共51个周期 \n",
    "    for loop in range(51): \n",
    "        #刚开始学习率比较大，后来慢慢变小 \n",
    "        sess.run(tf.assign(lr, 0.001 * (0.95 ** loop))) \n",
    "        #总共n_batch个批次\n",
    "        for batch in range(n_batch): \n",
    "            #获得一个批次 \n",
    "            batch_xs, batch_ys = mnist.train.next_batch(batch_size) \n",
    "            sess.run(train_step, \n",
    "                     feed_dict={x:batch_xs, y:batch_ys, keep_prob:0.8}\n",
    "                    ) \n",
    "        learning_rate = sess.run(lr) \n",
    "            #训练完一个周期后测试数据准确率 \n",
    "        accruacy = sess.run(accuracy, feed_dict={x:mnist.test.images, y:mnist.test.labels, keep_prob:1.0}) \n",
    "        print(\"Iterm\" + str(loop) + \", Testing Accuracy\" + str(accruacy)+ \", Learning_rate\" + str(learning_rate))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#经过测试，dropout只加一层的效果要好于两层，保存比例0.8，learnrate衰减速率为0.95\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
